pacman::p_load(tidyverse, wdman, jsonlite, rvest, xml2, RSelenium, here, crayon, here, magrittr)
library(RSelenium)
Paket 㤼㸱RSelenium㤼㸲 wurde unter R Version 3.4.3 erstelltWarning message:
In checkMatrixPackageVersion() : Package version inconsistency detected.
TMB was built with Matrix version 1.2.12
Current Matrix version is 1.2.11
Please re-install 'TMB' from source or ask CRAN for a binary version of 'TMB' matching CRAN's 'Matrix' package
rD
$client
browserName
1 chrome
id
1 b44485ef708363f81aefc2e486dde766
$server
Process Handle
command : C:\Users\Fabio\AppData\Local\Temp\Rtmpq0hjCh\file381c6113789b.bat
system id : 13812
state : running
url <- "https://www.idea.int/advanced-search?th=Direct%20Democracy%20Database"
checkboxes1 <- c("t46", "t47", "t48", "t49", "t50")
"//input[@value='",jj ,"'][@type='checkbox']"
Selenium message:unknown error: Element <input type="checkbox" name="questions" data-path="46" value="t46"> is not clickable at point (53, 672). Other element would receive the click: <div id="sliding-popup" class="sliding-popup-bottom" style="height: auto; width: 100%; bottom: 0px;">...</div>
(Session info: chrome=65.0.3325.146)
(Driver info: chromedriver=2.36.540470 (e522d04694c7ebea4ba8821272dbef4f9b818c91),platform=Windows NT 10.0.16299 x86_64)
Error: Summary: UnknownError
Detail: An unknown server-side error occurred while processing the command.
Further Details: run errorDetails method
library(rvest)
idea_html <- read_html("data/idea.html")
# labels <- idea_html %>%
# rvest::html_nodes(".country , .answer-value") %>%
# rvest::html_text()
#
# codes <- idea_html %>%
# rvest::html_nodes(".country , .answer-value") %>%
# rvest::html_attr("data-nid")
#
# idea_html %>%
# rvest::html_nodes(".fixed-header div") %>%
# rvest::html_text()
#
# data.frame(labels, codes)
# 9108 / 198
cntry <- idea_html %>%
rvest::html_nodes(".country") %>%
rvest::html_text() %>%
rep(each=46)
codes <- idea_html %>%
rvest::html_nodes(".answer-value") %>%
rvest::html_attr("data-nid") %>%
as.character()
labels <- readxl::read_xls("data/idea.xls") %>%
names() %>%
.[-1] %>%
rep(198)
val_labs <- idea_html %>%
rvest::html_nodes(".answer-value") %>%
rvest::html_text()
scrape_dat <- data.frame(labels, val_labs, cntry, codes, stringsAsFactors = F) %>%
mutate(val_labs = ifelse(val_labs == "", NA, val_labs)) %>%
mutate(urls = paste0("https://www.idea.int/node/", codes))
years <- 1776:2018 %>%
as.character() %>%
paste0(collapse = "|")
# texts <- read_html(scrape_dat$urls[1]) %>%
# html_nodes("#block-system-main") %>%
# html_text()#
# test <- scrape_dat %>%
# filter(codes %in% c("200518", "226549", "229780", "0")) %>%
# .[1:4,] %>%
# mutate(urls = ifelse(codes == "0", NA, urls))
scrape_dat %<>%
mutate(urls = ifelse(codes == "0", NA, urls))
source("helper_function.R")
length(lastyear)
start <- 1
end <- 9108
year_dat <- scraper_of_the_year(scrape_dat, start, end)
scraperei <- cbind(scrape_dat, year_dat)
save(scraperei, file = "data/scraperei.Rdata")
#
# str_extract_all("hello this is one date 1999, this wouldn't be a date 3456 but this is one 1890", years)
load("data/scraperei.Rdata")
scraperei %>%
filter(nyears == 2 & firstyear == lastyear) %>%
arrange(desc(nyears))
scraperei %>%
filter(nyears == 1) %>%
arrange(desc(nyears))
scraperei %>%
drop_na(firstyear)
pacman::p_load(tidyverse, wdman, jsonlite, rvest, xml2, RSelenium, here, crayon, here, magrittr, sjmisc)
library(rvest)
ref_name <- read_html("data/sudd.html") %>%
urls <- read_html("data/sudd.html") %>%
html_nodes(".leer a") %>%
html_attr("href")
dates <- read_html("data/sudd.html") %>%
html_nodes(".leer:nth-child(3)") %>%
html_text()
scrape_sudd <- data.frame(cntry, dates, ref_name, urls, stringsAsFactors = F)
read_html("data/sudd.html") %>%
html_nodes("td") %>%
html_table()
Error: html_name(x) == "table" is not TRUE
start <- 1
end <- 2813
source("helper_function.R")
sudd_dat <- scraper_of_the_dats(scrape_sudd, start, end) %>%
bind_rows()
[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=nl012018 of Country: Niederlande
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch012018 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch022018 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec012018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec022018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec032018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec042018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec052018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec062018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec072018 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=au012017 of Country: Australien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=it012017 of Country: Lombardei (Italien)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=br012017 of Country: Südbrasilien (Paraná, Santa Catarina, Rio Grande do Sul)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=es012017 of Country: Katalonien (Spanien)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=iq012017 of Country: Irakisch-Kurdistan (Irak)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=si012017 of Country: Slowenien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch062017 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch072017 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=cl012017 of Country: Osterinsel (Rapa Nui, Isla de Pascua), Chile
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=cl032017 of Country: Osterinsel (Rapa Nui, Isla de Pascua), Chile
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=mr012017 of Country: Mauretanien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve012017 of Country: Venezuela
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve022017 of Country: Venezuela
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ve032017 of Country: Venezuela
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=pr012017 of Country: Puerto Rico
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch042017 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ge012017 of Country: Südossetien (Georgien)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=hk012017 of Country: Hongkong
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=fm012017 of Country: Mikronesischer Staatenbund
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az012017 of Country: Berg-Karabach (Aserbaidschan)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ec012017 of Country: Ecuador
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch012017 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch032017 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=kg012016 of Country: Kirgisistan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=it022016 of Country: Italien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch132016 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd012016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd022016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd042016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd052016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd062016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=gd072016 of Country: Grenada
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=bg012016 of Country: Bulgarien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=bg032016 of Country: Bulgarien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ci012016 of Country: Elfenbeinküste
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=hu012016 of Country: Ungarn
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=co012016 of Country: Kolumbien
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=br012016 of Country: Südbrasilien (Paraná, Santa Catarina, Rio Grande do Sul)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az022016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az032016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az042016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az052016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az062016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az072016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az092016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az102016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az122016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az132016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az142016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az152016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az162016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az172016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az182016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az192016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az202016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az212016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az222016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az232016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az242016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az252016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az262016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az272016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az282016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=az292016 of Country: Aserbaidschan
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch102016 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ch112016 of Country: Schweiz
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=ba012016 of Country: Republika Srpska (Bosnien-Herzegowina)
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=li012016 of Country: Liechtenstein
[22m[49m[41m[1m
Checking URL: #: https://www.sudd.ch/event.php?lang=de&id=zm012016 of Country: Sambia
[22m[49m